import json
import queue
import re
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed

import requests
import unicodedata
from bs4 import BeautifulSoup
from tqdm import tqdm

from console_py.console import Console
# 从单独文件导入常量
from constants import RequestParam
from db import DBConfig


# 从古诗文网爬取拼音数据 https://www.gushiwen.cn/
class PoetryPinyinProcessor:
    def __init__(self, batch_size=1000, max_workers=10):
        self.db = DBConfig(db_type='remote')
        self.batch_size = batch_size  # 批量插入的大小
        self.max_workers = max_workers  # 最大线程数
        self.lock = threading.Lock()  # 线程锁
        self.progress = {
            'total': 0,
            'processed': 0,
            'success': 0,
            'failed': 0,
            'last_update': time.time()
        }
        self.insert_queue = queue.Queue()  # 插入队列
        self.stop_event = threading.Event()  # 停止事件
        self.console = Console()

    def get_poetry_data(self):
        """从数据库获取需要处理拼音的诗歌数据"""
        query = """
        SELECT id, content, ref_gushiwen_id 
        FROM chinese_poetry 
        WHERE ref_gushiwen_id IS NOT NULL AND collection='TSB' 
        """
        return self.db.execute_query(sql=query)

    def process_poetry(self, poetry):
        """处理单首诗歌"""
        poem_id = poetry['id']
        content_json = poetry['content']
        ref_gushiwen_id = poetry['ref_gushiwen_id']

        try:
            # 解析JSON内容
            content_lines = json.loads(content_json)

            # 获取拼音ID
            pinyin_id = self.get_pinyin_id(ref_gushiwen_id)
            if not pinyin_id:
                self.console.error(f"无法获取拼音ID for poem_id={poem_id}, ref_id={ref_gushiwen_id}")
                return False, []

            # 获取拼音数据
            pinyin_data = self.parse_pinyin_html(pinyin_id)
            if not pinyin_data:
                self.console.error(f"无法获取拼音数据 for poem_id={poem_id}")
                return False, []

            # 处理并返回拼音数据
            return True, self.process_pinyin_data(poem_id, content_lines, pinyin_data)

        except Exception as e:
            self.console.error(f"处理诗歌 {poem_id} 时出错: {str(e)}")
            return False, []

    def get_pinyin_id(self, page_id):
        """获取拼音ID"""
        url = f'https://www.gushiwen.cn/shiwenv_{page_id}.aspx'

        try:
            response = requests.get(url, headers=RequestParam.headers, timeout=10)
            response.raise_for_status()

            if response.status_code != 200:
                self.console.error(f"请求失败，状态码: {response.status_code}")
                return None

            soup = BeautifulSoup(response.text, 'html.parser')
            btn_id = f'btnPinyin{page_id}'
            btn_element = soup.find('a', {'id': btn_id})

            if btn_element:
                onclick_js = btn_element.get('onclick', '')
                match = re.search(
                    r'OnPinyin\d+\(\s*[\'"]?([^\'",]+)[\'"]?\s*,\s*[\'"]?([^\'",]+)[\'"]?\s*\)',
                    onclick_js
                )

                if match:
                    id2 = match.group(2)
                    return id2
                else:
                    possible_id2 = re.search(
                        r'OnPinyin\d+\(\s*[\'"]?' + re.escape(page_id) + r'[\'"]?\s*,\s*[\'"]?([^\'",]+)[\'"]?\s*\)',
                        response.text)
                    if possible_id2:
                        return possible_id2.group(1)
            else:
                script_elements = soup.find_all(
                    lambda tag: "onclick" in tag.attrs and f"OnPinyin" in tag.get("onclick", ""))
                for script_element in script_elements:
                    onclick_content = script_element["onclick"]
                    match = re.search(r'OnPinyin\d+\(\s*[\'"]?([^\'",]+)[\'"]?\s*,\s*[\'"]?([^\'",]+)[\'"]?\s*\)',
                                      onclick_content)
                    if match and match.group(1) == page_id:
                        return match.group(2)

        except requests.RequestException as e:
            self.console.error(f"网络请求失败: {e}")
        return None

    def parse_pinyin_html(self, pinyin_id):
        """解析拼音网页内容，支持多首诗分离"""
        url = f'https://www.gushiwen.cn/nocdn/ajaxshiwencont230427.aspx?id={pinyin_id}&value=yin'
        try:
            response = requests.get(url, headers=RequestParam.headers, timeout=10)
            response.raise_for_status()

            if response.status_code != 200:
                self.console.error(f"请求失败，状态码: {response.status_code}")
                return None

            soup = BeautifulSoup(response.text, 'html.parser')
            all_poems = []  # 存储所有诗的列表
            current_poem = []  # 当前正在解析的诗

            for container in soup.find_all('div', class_='pinyinContson'):
                # 查找容器内的所有<p>标签（每首诗通常在一个<p>中）
                p_tags = container.find_all('p')

                # 如果没有<p>标签，则整个容器视为一首诗
                if not p_tags:
                    poem_data = self.parse_poem_from_container(container)
                    if poem_data:
                        all_poems.append(poem_data)
                    continue

                # 处理每个<p>标签（每首独立的诗）
                for p_tag in p_tags:
                    poem_data = self.parse_poem_from_p_tag(p_tag)
                    if poem_data:
                        all_poems.append(poem_data)

            return all_poems
        except requests.RequestException as e:
            self.console.error(f"网络请求失败: {e}")
        except Exception as e:
            self.console.error(f"解析拼音时出错: {e}")
        return None

    def parse_poem_from_container(self, container):
        """从容器中解析一首诗"""
        poem_lines = []
        current_line = []
        in_comment = False

        for span in container.find_all('span'):
            hanzi_span = span.find(class_='hanzi')
            if not hanzi_span:
                continue

            h_text = hanzi_span.get_text(strip=True)
            p_text = span.find(class_='pinyin').get_text(strip=True) if span.find(class_='pinyin') else ''

            # 处理注释范围
            if h_text in ['(', '（']:
                in_comment = True
                continue
            elif h_text in [')', '）']:
                in_comment = False
                continue

            # 跳过注释内容
            if in_comment:
                continue

            # 检测换行（通过<br>标签）
            if span.find_parent('br'):
                if current_line:
                    poem_lines.append(current_line)
                    current_line = []

            # 添加字符到当前行
            current_line.append({
                'char': h_text,
                'pinyin': p_text
            })

        # 添加最后一行
        if current_line:
            poem_lines.append(current_line)

        return poem_lines

    def parse_poem_from_p_tag(self, p_tag):
        """从<p>标签中解析一首诗"""
        poem_lines = []
        current_line = []
        in_comment = False

        # 处理所有子节点
        for child in p_tag.children:
            # 处理<br>标签（行结束）
            if child.name == 'br' and current_line:
                poem_lines.append(current_line)
                current_line = []
                continue

            # 处理<span>标签
            if child.name == 'span':
                hanzi_span = child.find(class_='hanzi')
                if not hanzi_span:
                    continue

                h_text = hanzi_span.get_text(strip=True)
                p_text = child.find(class_='pinyin').get_text(strip=True) if child.find(class_='pinyin') else ''

                # 处理注释范围
                if h_text in ['(', '（']:
                    in_comment = True
                    continue
                elif h_text in [')', '）']:
                    in_comment = False
                    continue

                # 跳过注释内容
                if in_comment:
                    continue

                # 添加字符到当前行
                current_line.append({
                    'char': h_text,
                    'pinyin': p_text
                })

        # 添加最后一行
        if current_line:
            poem_lines.append(current_line)

        return poem_lines

    def process_pinyin_data(self, poem_id, content_lines, all_pinyin_data):
        """处理拼音数据并返回插入数据列表"""
        # 1. 找到与数据库内容最匹配的诗
        matched_poem = self.find_matching_poem(content_lines, all_pinyin_data)

        if not matched_poem:
            self.console.warning(f"警告: poem_id={poem_id} 未找到匹配的诗")
            return []

        # 2. 处理匹配到的诗
        return self.align_poem_lines(poem_id, content_lines, matched_poem)

    def find_matching_poem(self, content_lines, all_pinyin_data):
        """在多首诗中找出与数据库内容最匹配的那首"""
        # 准备数据库文本（去除标点）
        db_text = self.normalize_text(content_lines)

        best_match = None
        best_score = 0

        for poem in all_pinyin_data:
            # 将整首诗转换为字符串
            poem_text = self.poem_to_text(poem)
            # 归一化处理
            poem_text = self.normalize_text([poem_text])

            # 计算匹配分数
            score = self.calculate_match_score(db_text, poem_text)

            # 更新最佳匹配
            if score > best_score:
                best_match = poem
                best_score = score

        return best_match

    def poem_to_text(self, poem):
        """将整首诗转换为纯文本字符串"""
        text_lines = []
        for line in poem:
            text_lines.append(''.join(char_data['char'] for char_data in line))
        return ''.join(text_lines)

    def normalize_text(self, lines):
        """归一化文本（去除标点和空格）"""
        text = ''.join(lines)
        # 移除所有标点符号和空格
        return re.sub(r'[^\w]|_', '', text)

    def calculate_match_score(self, db_text, poem_text):
        """计算文本匹配分数"""
        # 简单实现：计算公共子序列长度
        score = 0
        i = 0
        for char in db_text:
            if i < len(poem_text) and poem_text[i] == char:
                score += 1
                i += 1
        return score

    def align_poem_lines(self, poem_id, content_lines, pinyin_data):
        """对齐处理：将数据库的分句与网络的整段进行匹配"""
        insert_data = []

        # 辅助函数：判断字符是否为标点符号或空格
        def is_punctuation(char):
            # Unicode分类中以"P"开头的表示标点符号
            if unicodedata.category(char).startswith('P'):
                return True
            # 检查常见的汉字标点
            chinese_punctuation = "，。？！；：．：；、-—~～（）《》【】〔〕"
            return char in chinese_punctuation or char.isspace()

        # 1. 准备数据库文本（分句）和网络文本（整段）
        # 数据库文本：按句存储的字符列表
        db_text_lines = [list(line) for line in content_lines]

        # 网络文本：将所有拼音数据合并为一个字符列表
        web_text_chars = []
        web_text_data = []  # 保存完整的字符数据
        for line in pinyin_data:
            for char_data in line:
                web_text_chars.append(char_data['char'])
                web_text_data.append(char_data)

        # 2. 对齐处理：将数据库的分句与网络的整段进行匹配
        matched_chars = 0
        skipped_chars = 0
        web_index = 0  # 网络文本的当前索引

        # 遍历数据库的每一句
        for line_idx, db_line in enumerate(db_text_lines):
            # 遍历数据库句中的每个字符
            for char_idx, db_char in enumerate(db_line):
                # 跳过标点符号
                if is_punctuation(db_char):
                    skipped_chars += 1
                    continue

                # 检查网络文本中是否有匹配的字符
                found_match = False

                # 在剩余的网络文本中查找匹配的字符
                max_search = min(10, len(web_text_chars) - web_index)
                for offset in range(max_search):
                    web_char = web_text_chars[web_index + offset]

                    # 如果找到匹配字符
                    if db_char == web_char:
                        # 获取对应的拼音数据
                        char_data = web_text_data[web_index + offset]
                        insert_data.append((
                            poem_id,
                            line_idx,
                            char_idx,
                            db_char,
                            char_data['pinyin']
                        ))
                        matched_chars += 1
                        found_match = True
                        # 更新网络索引到这个字符之后
                        web_index += offset + 1
                        break

                # 如果没有找到匹配字符
                if not found_match:
                    skipped_chars += 1
                    self.console.warning(f"警告: poem_id={poem_id} 行{line_idx} 字{char_idx} 字符未找到匹配: {db_char}")

        return insert_data

    def insert_worker(self):
        """批量插入工作线程"""
        insert_query = """
        INSERT INTO chinese_poetry_pinyin 
        (poem_id, line_index, char_index, original_char, pinyin) 
        VALUES (%s, %s, %s, %s, %s)
        """

        while not self.stop_event.is_set() or not self.insert_queue.empty():
            try:
                # 从队列中获取数据
                data_batch = []
                while len(data_batch) < self.batch_size:
                    try:
                        data = self.insert_queue.get(timeout=1)
                        data_batch.append(data)
                    except queue.Empty:
                        if self.stop_event.is_set():
                            break

                if data_batch:
                    # 执行批量插入
                    self.db.batch_execute(insert_query, data_batch)
                    self.console.log(f"批量插入 {len(data_batch)} 条记录")
            except Exception as e:
                self.console.error(f"批量插入出错: {str(e)}")

    def update_progress(self, success):
        """更新进度信息"""
        with self.lock:
            self.progress['processed'] += 1
            if success:
                self.progress['success'] += 1
            else:
                self.progress['failed'] += 1

            # 每处理100首诗或每5秒更新一次进度
            current_time = time.time()
            if (self.progress['processed'] % 100 == 0 or
                    current_time - self.progress['last_update'] > 5):
                self.progress['last_update'] = current_time
                self.print_progress()

    def print_progress(self):
        """打印当前进度"""
        processed = self.progress['processed']
        total = self.progress['total']
        success = self.progress['success']
        failed = self.progress['failed']
        percentage = (processed / total) * 100 if total > 0 else 0

        self.console.log(f"\r进度: {processed}/{total} ({percentage:.2f}%) | "
                         f"成功: {success} | 失败: {failed} | "
                         f"队列大小: {self.insert_queue.qsize()}", end="")
        sys.stdout.flush()

    def run(self):
        """主处理循环"""
        poetry_list = self.get_poetry_data()
        self.progress['total'] = len(poetry_list)
        self.console.log(f"找到 {len(poetry_list)} 首诗歌需要处理")

        # 启动插入工作线程
        insert_thread = threading.Thread(target=self.insert_worker, daemon=True)
        insert_thread.start()

        try:
            # 使用线程池处理诗歌
            with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
                futures = {executor.submit(self.process_poetry, poetry): poetry for poetry in poetry_list}

                for future in tqdm(as_completed(futures), total=len(poetry_list), desc="处理诗歌"):
                    poetry = futures[future]
                    try:
                        success, insert_data = future.result()
                        self.update_progress(success)

                        # 将数据放入插入队列
                        if insert_data:
                            for data in insert_data:
                                self.insert_queue.put(data)
                    except Exception as e:
                        self.console.error(f"处理诗歌出错: {str(e)}")
                        self.update_progress(False)

            # 等待所有插入完成
            self.stop_event.set()
            insert_thread.join()

            self.console.log("\n处理完成!")
            self.print_progress()
        except KeyboardInterrupt:
            self.console.warning("\n用户中断，停止处理...")
            self.stop_event.set()
            insert_thread.join()
        except Exception as e:
            self.console.error(f"处理过程中出错: {str(e)}")
            self.stop_event.set()
            insert_thread.join()


if __name__ == "__main__":
    # 可以在这里调整批量大小和线程数
    processor = PoetryPinyinProcessor(batch_size=1000, max_workers=10)
    processor.run()
